In this project, we will be training on the Oxford-iiit-pets dataset which has 37 categoies: 25 categories of dogs and 12 categories of cats. We pass our images through the ResNet50 model (without the final output classification layer).

We start by loading libraries relevant to our project.

library(readr)
library(ggplot2)
library(dplyr)
library(methods)
library(stringi)
library(keras)

We now load the ResNet50 model and grab the second last layer from the model.

resnet50 <- application_resnet50(weights = 'imagenet', include_top = TRUE)
model_embed <- keras_model(inputs = resnet50$input,
                           outputs = get_layer(resnet50, 'avg_pool')$output)

We load all of our data from our csv and rds files which were constructed in another class:

pets <- read_csv("my-image-data.csv")
## Parsed with column specification:
## cols(
##   obs_id = col_character(),
##   train_id = col_character(),
##   class = col_double(),
##   class_name = col_character(),
##   path_to_image = col_character()
## )
x69 <- read_rds("final_project.rds")

We now produce our embeddings to get our X and y matrices:

X <- t(apply(x69, 1, cbind))
y <- pets$class

class_names <- levels(factor(pets$class_name))

We create our training dataset from our matrices, where the training/validation split is 60/40 respectively:

X_train <- X[pets$train_id == "train",]          
y_train <- to_categorical(pets$class[pets$train_id == "train"])

To see a sample of some of the images in the dataset:

set.seed(1)
par(mfrow = c(2, 2))

samples = sample(0:37, 5)
for (j in samples) {
  id <- sample(which(y == j), 4)
  
  for (i in id) {
    par(mar = rep(0, 4L))
    plot(0,0,xlim=c(0,1),ylim=c(0,1),axes= FALSE,type = "n")
    Z <- image_to_array(image_load(pets$path_to_image[i], target_size = c(224,224)))
    rasterImage(Z /255,0,0,1,1)
    text(0.5, 0.1, label = class_names[y[i] + 1L], col = "red", cex=2)
  }
}

We create a keras model with multiple layers added:

model <- keras_model_sequential()
model %>%
  layer_dense(units = 256, input_shape = ncol(X_train)) %>%
  layer_activation(activation = "relu") %>%
  layer_dropout(rate = 0.5) %>%

  layer_dense(units = 256) %>%
  layer_activation(activation = "relu") %>%
  layer_dropout(rate = 0.5) %>%

  layer_dense(units = ncol(y_train)) %>%
  layer_activation(activation = "softmax")

We compile our model:

model %>% compile(loss = 'categorical_crossentropy',
                  optimizer = optimizer_rmsprop(lr = 0.001 / 2),
                  metrics = c('accuracy'))

We fit our model to our dataset:

history <- model %>% fit(X_train, y_train, epochs = 20)

plot(history)

We find our predictions from this model:

y_pred <- predict_classes(model, X)
tapply(y == y_pred, pets$train_id, mean)
##     train     valid 
## 0.9944367 0.9036857

We see that the model overfits the data (99.4% training accuracy) but the validation accuracy is fairly high at 90.2%.

We also print the confusion matrix:

table(value = class_names[y + 1L], prediction = class_names[y_pred + 1L], pets$train_id)
## , ,  = train
## 
##                             prediction
## value                        Abyssinian American bulldog
##   Abyssinian                        120                0
##   American bulldog                    0              118
##   American Pit Bull Terrier           0                0
##   Basset Hound                        0                0
##   Beagle                              0                0
##   Bengal                              0                0
##   Birman                              0                0
##   Bombay                              0                0
##   Boxer                               0                1
##   British Shorthair                   0                0
##   Chihuahua                           0                0
##   Egyptian Mau                        0                0
##   English Setter                      0                0
##   German shorthaired                  0                0
##   Great Pyreness                      0                0
##   Havanese                            0                0
##   Japanese Chin                       0                0
##   Keeshond                            0                0
##   Leonberger                          0                0
##   Maine Coon                          0                0
##   Miniature Pinscher                  0                0
##   Newfoundland                        0                0
##   Persian                             0                0
##   Pomeranian                          0                0
##   Pug                                 0                0
##   Ragdoll                             0                0
##   Russian Blue                        0                0
##   Saint Bernard                       0                0
##   Samoyed                             0                0
##   Scottish Terrier                    0                0
##   Shiba Inu                           0                0
##   Siamese                             0                0
##   Sphynx                              0                0
##   Staffordshire Bull Terrier          0                0
##   Wheaten Terrier                     0                0
##   Yorkshire Terrier                   0                0
##                             prediction
## value                        American Pit Bull Terrier Basset Hound Beagle
##   Abyssinian                                         0            0      0
##   American bulldog                                   1            0      0
##   American Pit Bull Terrier                        118            0      0
##   Basset Hound                                       0          120      0
##   Beagle                                             0            2    118
##   Bengal                                             0            0      0
##   Birman                                             0            0      0
##   Bombay                                             0            0      0
##   Boxer                                              0            0      0
##   British Shorthair                                  0            0      0
##   Chihuahua                                          0            0      0
##   Egyptian Mau                                       0            0      0
##   English Setter                                     0            0      0
##   German shorthaired                                 0            0      1
##   Great Pyreness                                     0            0      0
##   Havanese                                           0            0      0
##   Japanese Chin                                      0            0      0
##   Keeshond                                           0            0      0
##   Leonberger                                         0            0      0
##   Maine Coon                                         0            0      0
##   Miniature Pinscher                                 0            0      0
##   Newfoundland                                       0            0      0
##   Persian                                            0            0      0
##   Pomeranian                                         0            0      0
##   Pug                                                0            0      0
##   Ragdoll                                            0            0      0
##   Russian Blue                                       0            0      0
##   Saint Bernard                                      0            0      0
##   Samoyed                                            0            0      0
##   Scottish Terrier                                   0            0      0
##   Shiba Inu                                          0            0      0
##   Siamese                                            0            0      0
##   Sphynx                                             0            0      0
##   Staffordshire Bull Terrier                         1            0      0
##   Wheaten Terrier                                    0            0      0
##   Yorkshire Terrier                                  0            0      0
##                             prediction
## value                        Bengal Birman Bombay Boxer British Shorthair
##   Abyssinian                      0      0      0     0                 0
##   American bulldog                0      0      0     0                 0
##   American Pit Bull Terrier       0      0      0     0                 0
##   Basset Hound                    0      0      0     0                 0
##   Beagle                          0      0      0     0                 0
##   Bengal                        120      0      0     0                 0
##   Birman                          0    120      0     0                 0
##   Bombay                          0      0    120     0                 0
##   Boxer                           0      0      0   119                 0
##   British Shorthair               0      0      0     0               120
##   Chihuahua                       0      0      0     0                 0
##   Egyptian Mau                    1      0      0     0                 0
##   English Setter                  0      0      0     0                 0
##   German shorthaired              0      0      0     0                 0
##   Great Pyreness                  0      0      0     0                 0
##   Havanese                        0      0      0     0                 0
##   Japanese Chin                   0      0      0     0                 0
##   Keeshond                        0      0      0     0                 0
##   Leonberger                      0      0      0     0                 0
##   Maine Coon                      0      0      0     0                 0
##   Miniature Pinscher              0      0      0     0                 0
##   Newfoundland                    0      0      0     0                 0
##   Persian                         0      0      0     0                 0
##   Pomeranian                      0      0      0     0                 0
##   Pug                             0      0      0     0                 0
##   Ragdoll                         0      8      0     0                 0
##   Russian Blue                    0      0      0     0                 1
##   Saint Bernard                   0      0      0     0                 0
##   Samoyed                         0      0      0     0                 0
##   Scottish Terrier                0      0      0     0                 0
##   Shiba Inu                       0      0      0     0                 0
##   Siamese                         0      0      0     0                 0
##   Sphynx                          0      0      0     0                 0
##   Staffordshire Bull Terrier      0      0      0     0                 0
##   Wheaten Terrier                 0      0      0     0                 0
##   Yorkshire Terrier               0      0      0     0                 0
##                             prediction
## value                        Chihuahua Egyptian Mau English Setter
##   Abyssinian                         0            0              0
##   American bulldog                   0            0              0
##   American Pit Bull Terrier          0            0              0
##   Basset Hound                       0            0              0
##   Beagle                             0            0              0
##   Bengal                             0            0              0
##   Birman                             0            0              0
##   Bombay                             0            0              0
##   Boxer                              0            0              0
##   British Shorthair                  0            0              0
##   Chihuahua                        119            0              0
##   Egyptian Mau                       0          119              0
##   English Setter                     0            0            120
##   German shorthaired                 0            0              0
##   Great Pyreness                     0            0              0
##   Havanese                           0            0              0
##   Japanese Chin                      0            0              0
##   Keeshond                           0            0              0
##   Leonberger                         0            0              0
##   Maine Coon                         0            0              0
##   Miniature Pinscher                 0            0              0
##   Newfoundland                       0            0              0
##   Persian                            0            0              0
##   Pomeranian                         0            0              0
##   Pug                                0            0              0
##   Ragdoll                            0            0              0
##   Russian Blue                       0            0              0
##   Saint Bernard                      0            0              0
##   Samoyed                            0            0              0
##   Scottish Terrier                   0            0              0
##   Shiba Inu                          0            0              0
##   Siamese                            0            0              0
##   Sphynx                             0            0              0
##   Staffordshire Bull Terrier         0            0              0
##   Wheaten Terrier                    0            0              0
##   Yorkshire Terrier                  0            0              0
##                             prediction
## value                        German shorthaired Great Pyreness Havanese
##   Abyssinian                                  0              0        0
##   American bulldog                            0              0        0
##   American Pit Bull Terrier                   0              0        0
##   Basset Hound                                0              0        0
##   Beagle                                      0              0        0
##   Bengal                                      0              0        0
##   Birman                                      0              0        0
##   Bombay                                      0              0        0
##   Boxer                                       0              0        0
##   British Shorthair                           0              0        0
##   Chihuahua                                   0              0        0
##   Egyptian Mau                                0              0        0
##   English Setter                              0              0        0
##   German shorthaired                        119              0        0
##   Great Pyreness                              0            120        0
##   Havanese                                    0              0      119
##   Japanese Chin                               0              0        0
##   Keeshond                                    0              0        0
##   Leonberger                                  0              0        0
##   Maine Coon                                  0              0        0
##   Miniature Pinscher                          0              0        0
##   Newfoundland                                0              0        0
##   Persian                                     0              0        0
##   Pomeranian                                  0              0        0
##   Pug                                         0              0        0
##   Ragdoll                                     0              0        0
##   Russian Blue                                0              0        0
##   Saint Bernard                               0              0        0
##   Samoyed                                     0              0        0
##   Scottish Terrier                            0              0        0
##   Shiba Inu                                   0              0        0
##   Siamese                                     0              0        0
##   Sphynx                                      0              0        0
##   Staffordshire Bull Terrier                  0              0        0
##   Wheaten Terrier                             0              0        0
##   Yorkshire Terrier                           0              0        0
##                             prediction
## value                        Japanese Chin Keeshond Leonberger Maine Coon
##   Abyssinian                             0        0          0          0
##   American bulldog                       0        0          0          0
##   American Pit Bull Terrier              0        0          0          0
##   Basset Hound                           0        0          0          0
##   Beagle                                 0        0          0          0
##   Bengal                                 0        0          0          0
##   Birman                                 0        0          0          0
##   Bombay                                 0        0          0          0
##   Boxer                                  0        0          0          0
##   British Shorthair                      0        0          0          0
##   Chihuahua                              0        0          0          0
##   Egyptian Mau                           0        0          0          0
##   English Setter                         0        0          0          0
##   German shorthaired                     0        0          0          0
##   Great Pyreness                         0        0          0          0
##   Havanese                               0        0          0          0
##   Japanese Chin                        119        0          0          0
##   Keeshond                               0      120          0          0
##   Leonberger                             0        0        120          0
##   Maine Coon                             0        0          0        120
##   Miniature Pinscher                     0        0          0          0
##   Newfoundland                           0        0          0          0
##   Persian                                0        0          0          0
##   Pomeranian                             0        0          0          0
##   Pug                                    0        0          0          0
##   Ragdoll                                0        0          0          0
##   Russian Blue                           0        0          0          0
##   Saint Bernard                          1        0          0          0
##   Samoyed                                0        0          0          0
##   Scottish Terrier                       0        0          0          0
##   Shiba Inu                              0        0          0          0
##   Siamese                                0        0          0          0
##   Sphynx                                 0        0          0          0
##   Staffordshire Bull Terrier             0        0          0          0
##   Wheaten Terrier                        0        0          0          0
##   Yorkshire Terrier                      0        0          0          0
##                             prediction
## value                        Miniature Pinscher Newfoundland Persian
##   Abyssinian                                  0            0       0
##   American bulldog                            0            0       0
##   American Pit Bull Terrier                   0            0       0
##   Basset Hound                                0            0       0
##   Beagle                                      0            0       0
##   Bengal                                      0            0       0
##   Birman                                      0            0       0
##   Bombay                                      0            0       0
##   Boxer                                       0            0       0
##   British Shorthair                           0            0       0
##   Chihuahua                                   1            0       0
##   Egyptian Mau                                0            0       0
##   English Setter                              0            0       0
##   German shorthaired                          0            0       0
##   Great Pyreness                              0            0       0
##   Havanese                                    0            1       0
##   Japanese Chin                               0            0       0
##   Keeshond                                    0            0       0
##   Leonberger                                  0            0       0
##   Maine Coon                                  0            0       0
##   Miniature Pinscher                        120            0       0
##   Newfoundland                                0          120       0
##   Persian                                     0            0     119
##   Pomeranian                                  0            0       0
##   Pug                                         0            0       0
##   Ragdoll                                     0            0       0
##   Russian Blue                                0            0       0
##   Saint Bernard                               0            0       0
##   Samoyed                                     0            0       0
##   Scottish Terrier                            0            0       0
##   Shiba Inu                                   0            0       0
##   Siamese                                     0            0       0
##   Sphynx                                      0            0       0
##   Staffordshire Bull Terrier                  0            0       0
##   Wheaten Terrier                             0            0       0
##   Yorkshire Terrier                           0            0       0
##                             prediction
## value                        Pomeranian Pug Ragdoll Russian Blue
##   Abyssinian                          0   0       0            0
##   American bulldog                    0   0       0            0
##   American Pit Bull Terrier           0   0       0            0
##   Basset Hound                        0   0       0            0
##   Beagle                              0   0       0            0
##   Bengal                              0   0       0            0
##   Birman                              0   0       0            0
##   Bombay                              0   0       0            0
##   Boxer                               0   0       0            0
##   British Shorthair                   0   0       0            0
##   Chihuahua                           0   0       0            0
##   Egyptian Mau                        0   0       0            0
##   English Setter                      0   0       0            0
##   German shorthaired                  0   0       0            0
##   Great Pyreness                      0   0       0            0
##   Havanese                            0   0       0            0
##   Japanese Chin                       1   0       0            0
##   Keeshond                            0   0       0            0
##   Leonberger                          0   0       0            0
##   Maine Coon                          0   0       0            0
##   Miniature Pinscher                  0   0       0            0
##   Newfoundland                        0   0       0            0
##   Persian                             0   0       1            0
##   Pomeranian                        120   0       0            0
##   Pug                                 0 120       0            0
##   Ragdoll                             0   0     112            0
##   Russian Blue                        0   0       0          119
##   Saint Bernard                       0   0       0            0
##   Samoyed                             0   0       0            0
##   Scottish Terrier                    0   0       0            0
##   Shiba Inu                           0   0       0            0
##   Siamese                             0   0       0            0
##   Sphynx                              0   0       0            0
##   Staffordshire Bull Terrier          0   0       0            0
##   Wheaten Terrier                     0   0       0            0
##   Yorkshire Terrier                   0   0       0            0
##                             prediction
## value                        Saint Bernard Samoyed Scottish Terrier
##   Abyssinian                             0       0                0
##   American bulldog                       0       0                0
##   American Pit Bull Terrier              0       0                0
##   Basset Hound                           0       0                0
##   Beagle                                 0       0                0
##   Bengal                                 0       0                0
##   Birman                                 0       0                0
##   Bombay                                 0       0                0
##   Boxer                                  0       0                0
##   British Shorthair                      0       0                0
##   Chihuahua                              0       0                0
##   Egyptian Mau                           0       0                0
##   English Setter                         0       0                0
##   German shorthaired                     0       0                0
##   Great Pyreness                         0       0                0
##   Havanese                               0       0                0
##   Japanese Chin                          0       0                0
##   Keeshond                               0       0                0
##   Leonberger                             0       0                0
##   Maine Coon                             0       0                0
##   Miniature Pinscher                     0       0                0
##   Newfoundland                           0       0                0
##   Persian                                0       0                0
##   Pomeranian                             0       0                0
##   Pug                                    0       0                0
##   Ragdoll                                0       0                0
##   Russian Blue                           0       0                0
##   Saint Bernard                        119       0                0
##   Samoyed                                0     120                0
##   Scottish Terrier                       0       0              119
##   Shiba Inu                              0       0                0
##   Siamese                                0       0                0
##   Sphynx                                 0       0                0
##   Staffordshire Bull Terrier             0       0                0
##   Wheaten Terrier                        0       0                0
##   Yorkshire Terrier                      0       0                0
##                             prediction
## value                        Shiba Inu Siamese Sphynx
##   Abyssinian                         0       0      0
##   American bulldog                   0       0      0
##   American Pit Bull Terrier          0       0      0
##   Basset Hound                       0       0      0
##   Beagle                             0       0      0
##   Bengal                             0       0      0
##   Birman                             0       0      0
##   Bombay                             0       0      0
##   Boxer                              0       0      0
##   British Shorthair                  0       0      0
##   Chihuahua                          0       0      0
##   Egyptian Mau                       0       0      0
##   English Setter                     0       0      0
##   German shorthaired                 0       0      0
##   Great Pyreness                     0       0      0
##   Havanese                           0       0      0
##   Japanese Chin                      0       0      0
##   Keeshond                           0       0      0
##   Leonberger                         0       0      0
##   Maine Coon                         0       0      0
##   Miniature Pinscher                 0       0      0
##   Newfoundland                       0       0      0
##   Persian                            0       0      0
##   Pomeranian                         0       0      0
##   Pug                                0       0      0
##   Ragdoll                            0       0      0
##   Russian Blue                       0       0      0
##   Saint Bernard                      0       0      0
##   Samoyed                            0       0      0
##   Scottish Terrier                   0       0      0
##   Shiba Inu                        120       0      0
##   Siamese                            0     120      0
##   Sphynx                             0       0    120
##   Staffordshire Bull Terrier         0       0      0
##   Wheaten Terrier                    0       0      0
##   Yorkshire Terrier                  0       0      0
##                             prediction
## value                        Staffordshire Bull Terrier Wheaten Terrier
##   Abyssinian                                          0               0
##   American bulldog                                    1               0
##   American Pit Bull Terrier                           2               0
##   Basset Hound                                        0               0
##   Beagle                                              0               0
##   Bengal                                              0               0
##   Birman                                              0               0
##   Bombay                                              0               0
##   Boxer                                               0               0
##   British Shorthair                                   0               0
##   Chihuahua                                           0               0
##   Egyptian Mau                                        0               0
##   English Setter                                      0               0
##   German shorthaired                                  0               0
##   Great Pyreness                                      0               0
##   Havanese                                            0               0
##   Japanese Chin                                       0               0
##   Keeshond                                            0               0
##   Leonberger                                          0               0
##   Maine Coon                                          0               0
##   Miniature Pinscher                                  0               0
##   Newfoundland                                        0               0
##   Persian                                             0               0
##   Pomeranian                                          0               0
##   Pug                                                 0               0
##   Ragdoll                                             0               0
##   Russian Blue                                        0               0
##   Saint Bernard                                       0               0
##   Samoyed                                             0               0
##   Scottish Terrier                                    0               0
##   Shiba Inu                                           0               0
##   Siamese                                             0               0
##   Sphynx                                              0               0
##   Staffordshire Bull Terrier                        114               0
##   Wheaten Terrier                                     0             120
##   Yorkshire Terrier                                   0               0
##                             prediction
## value                        Yorkshire Terrier
##   Abyssinian                                 0
##   American bulldog                           0
##   American Pit Bull Terrier                  0
##   Basset Hound                               0
##   Beagle                                     0
##   Bengal                                     0
##   Birman                                     0
##   Bombay                                     0
##   Boxer                                      0
##   British Shorthair                          0
##   Chihuahua                                  0
##   Egyptian Mau                               0
##   English Setter                             0
##   German shorthaired                         0
##   Great Pyreness                             0
##   Havanese                                   0
##   Japanese Chin                              0
##   Keeshond                                   0
##   Leonberger                                 0
##   Maine Coon                                 0
##   Miniature Pinscher                         0
##   Newfoundland                               0
##   Persian                                    0
##   Pomeranian                                 0
##   Pug                                        0
##   Ragdoll                                    0
##   Russian Blue                               0
##   Saint Bernard                              0
##   Samoyed                                    0
##   Scottish Terrier                           0
##   Shiba Inu                                  0
##   Siamese                                    0
##   Sphynx                                     0
##   Staffordshire Bull Terrier                 0
##   Wheaten Terrier                            0
##   Yorkshire Terrier                        120
## 
## , ,  = valid
## 
##                             prediction
## value                        Abyssinian American bulldog
##   Abyssinian                         72                0
##   American bulldog                    0               67
##   American Pit Bull Terrier           0                3
##   Basset Hound                        0                0
##   Beagle                              0                1
##   Bengal                              4                0
##   Birman                              0                0
##   Bombay                              0                0
##   Boxer                               0                3
##   British Shorthair                   1                0
##   Chihuahua                           0                0
##   Egyptian Mau                        5                0
##   English Setter                      0                0
##   German shorthaired                  0                1
##   Great Pyreness                      0                0
##   Havanese                            0                0
##   Japanese Chin                       0                0
##   Keeshond                            0                0
##   Leonberger                          0                0
##   Maine Coon                          1                0
##   Miniature Pinscher                  0                0
##   Newfoundland                        0                0
##   Persian                             0                0
##   Pomeranian                          0                0
##   Pug                                 0                2
##   Ragdoll                             0                0
##   Russian Blue                        1                0
##   Saint Bernard                       0                0
##   Samoyed                             0                0
##   Scottish Terrier                    0                0
##   Shiba Inu                           0                0
##   Siamese                             1                0
##   Sphynx                              0                0
##   Staffordshire Bull Terrier          0                5
##   Wheaten Terrier                     0                1
##   Yorkshire Terrier                   0                0
##                             prediction
## value                        American Pit Bull Terrier Basset Hound Beagle
##   Abyssinian                                         0            0      0
##   American bulldog                                   9            0      0
##   American Pit Bull Terrier                         59            2      0
##   Basset Hound                                       1           78      1
##   Beagle                                             1           12     61
##   Bengal                                             0            1      0
##   Birman                                             0            0      0
##   Bombay                                             0            0      0
##   Boxer                                              2            1      0
##   British Shorthair                                  0            0      0
##   Chihuahua                                          0            1      1
##   Egyptian Mau                                       0            0      0
##   English Setter                                     0            0      0
##   German shorthaired                                 1            0      1
##   Great Pyreness                                     0            0      0
##   Havanese                                           0            0      0
##   Japanese Chin                                      0            0      0
##   Keeshond                                           0            0      0
##   Leonberger                                         0            0      0
##   Maine Coon                                         0            0      0
##   Miniature Pinscher                                 2            0      0
##   Newfoundland                                       0            0      0
##   Persian                                            0            0      0
##   Pomeranian                                         0            0      0
##   Pug                                                1            0      0
##   Ragdoll                                            0            0      0
##   Russian Blue                                       0            0      0
##   Saint Bernard                                      0            0      0
##   Samoyed                                            0            0      0
##   Scottish Terrier                                   0            0      0
##   Shiba Inu                                          0            0      0
##   Siamese                                            0            0      1
##   Sphynx                                             0            0      0
##   Staffordshire Bull Terrier                         9            1      1
##   Wheaten Terrier                                    0            0      0
##   Yorkshire Terrier                                  0            0      0
##                             prediction
## value                        Bengal Birman Bombay Boxer British Shorthair
##   Abyssinian                      3      0      0     0                 0
##   American bulldog                0      0      0     1                 0
##   American Pit Bull Terrier       0      0      0     2                 0
##   Basset Hound                    0      0      0     0                 0
##   Beagle                          0      0      0     0                 0
##   Bengal                         68      0      0     0                 0
##   Birman                          0     73      0     0                 0
##   Bombay                          0      0     79     0                 0
##   Boxer                           0      0      0    72                 0
##   British Shorthair               0      0      0     0                69
##   Chihuahua                       0      0      0     0                 0
##   Egyptian Mau                   11      0      1     0                 0
##   English Setter                  0      0      0     0                 0
##   German shorthaired              0      0      0     0                 0
##   Great Pyreness                  0      0      0     0                 0
##   Havanese                        0      0      0     0                 0
##   Japanese Chin                   0      0      0     0                 0
##   Keeshond                        0      0      0     0                 0
##   Leonberger                      0      0      0     1                 0
##   Maine Coon                      2      0      1     0                 0
##   Miniature Pinscher              0      0      0     0                 0
##   Newfoundland                    0      0      0     0                 0
##   Persian                         0      0      1     0                 1
##   Pomeranian                      0      0      0     0                 0
##   Pug                             0      0      0     1                 0
##   Ragdoll                         0     18      0     0                 0
##   Russian Blue                    0      0      3     0                 5
##   Saint Bernard                   0      0      0     2                 0
##   Samoyed                         0      0      0     0                 0
##   Scottish Terrier                0      0      0     0                 0
##   Shiba Inu                       0      0      0     0                 0
##   Siamese                         0     16      0     0                 0
##   Sphynx                          0      0      0     0                 0
##   Staffordshire Bull Terrier      0      0      0     0                 0
##   Wheaten Terrier                 0      1      0     0                 0
##   Yorkshire Terrier               0      0      0     0                 0
##                             prediction
## value                        Chihuahua Egyptian Mau English Setter
##   Abyssinian                         0            1              0
##   American bulldog                   0            0              0
##   American Pit Bull Terrier          1            0              0
##   Basset Hound                       0            0              0
##   Beagle                             0            0              1
##   Bengal                             0            4              0
##   Birman                             0            0              0
##   Bombay                             0            0              0
##   Boxer                              0            0              0
##   British Shorthair                  0            0              0
##   Chihuahua                         69            0              0
##   Egyptian Mau                       0           62              0
##   English Setter                     0            0             77
##   German shorthaired                 0            0              0
##   Great Pyreness                     0            0              0
##   Havanese                           0            0              0
##   Japanese Chin                      0            0              0
##   Keeshond                           0            0              0
##   Leonberger                         0            0              0
##   Maine Coon                         0            0              0
##   Miniature Pinscher                 3            0              0
##   Newfoundland                       0            0              0
##   Persian                            0            0              0
##   Pomeranian                         0            0              0
##   Pug                                0            0              0
##   Ragdoll                            0            0              0
##   Russian Blue                       0            0              0
##   Saint Bernard                      0            0              0
##   Samoyed                            0            0              0
##   Scottish Terrier                   0            0              0
##   Shiba Inu                          0            0              0
##   Siamese                            1            0              0
##   Sphynx                             1            0              0
##   Staffordshire Bull Terrier         0            0              0
##   Wheaten Terrier                    0            0              0
##   Yorkshire Terrier                  0            0              0
##                             prediction
## value                        German shorthaired Great Pyreness Havanese
##   Abyssinian                                  0              0        0
##   American bulldog                            0              1        0
##   American Pit Bull Terrier                   3              0        0
##   Basset Hound                                0              0        0
##   Beagle                                      1              1        0
##   Bengal                                      0              0        0
##   Birman                                      0              0        0
##   Bombay                                      0              0        0
##   Boxer                                       0              0        0
##   British Shorthair                           0              0        0
##   Chihuahua                                   0              0        0
##   Egyptian Mau                                0              0        0
##   English Setter                              1              1        0
##   German shorthaired                         76              1        0
##   Great Pyreness                              0             80        0
##   Havanese                                    0              0       74
##   Japanese Chin                               0              0        0
##   Keeshond                                    0              0        0
##   Leonberger                                  0              0        0
##   Maine Coon                                  0              0        0
##   Miniature Pinscher                          0              0        0
##   Newfoundland                                0              0        0
##   Persian                                     0              0        0
##   Pomeranian                                  0              0        0
##   Pug                                         0              0        0
##   Ragdoll                                     0              0        0
##   Russian Blue                                0              0        0
##   Saint Bernard                               0              0        0
##   Samoyed                                     0              5        0
##   Scottish Terrier                            0              0        0
##   Shiba Inu                                   0              0        0
##   Siamese                                     0              0        0
##   Sphynx                                      0              0        0
##   Staffordshire Bull Terrier                  0              0        0
##   Wheaten Terrier                             1              0        1
##   Yorkshire Terrier                           0              0        1
##                             prediction
## value                        Japanese Chin Keeshond Leonberger Maine Coon
##   Abyssinian                             0        0          0          0
##   American bulldog                       0        0          0          0
##   American Pit Bull Terrier              0        0          1          0
##   Basset Hound                           0        0          0          0
##   Beagle                                 0        0          0          0
##   Bengal                                 0        0          0          1
##   Birman                                 0        0          0          0
##   Bombay                                 0        0          0          0
##   Boxer                                  0        0          1          0
##   British Shorthair                      0        0          0          0
##   Chihuahua                              0        0          1          0
##   Egyptian Mau                           0        0          0          0
##   English Setter                         0        0          0          0
##   German shorthaired                     0        0          0          0
##   Great Pyreness                         0        0          0          0
##   Havanese                               0        0          0          0
##   Japanese Chin                         80        0          0          0
##   Keeshond                               0       79          1          0
##   Leonberger                             0        1         78          0
##   Maine Coon                             0        0          0         66
##   Miniature Pinscher                     0        0          0          0
##   Newfoundland                           0        0          1          0
##   Persian                                0        0          0          1
##   Pomeranian                             0        0          0          0
##   Pug                                    0        0          0          0
##   Ragdoll                                1        0          0          0
##   Russian Blue                           0        0          0          0
##   Saint Bernard                          0        0          0          0
##   Samoyed                                0        0          0          0
##   Scottish Terrier                       0        0          0          0
##   Shiba Inu                              0        0          0          0
##   Siamese                                0        0          1          0
##   Sphynx                                 0        0          0          0
##   Staffordshire Bull Terrier             0        0          0          0
##   Wheaten Terrier                        0        0          0          0
##   Yorkshire Terrier                      2        0          0          0
##                             prediction
## value                        Miniature Pinscher Newfoundland Persian
##   Abyssinian                                  1            0       0
##   American bulldog                            0            0       0
##   American Pit Bull Terrier                   1            0       0
##   Basset Hound                                0            0       0
##   Beagle                                      0            0       0
##   Bengal                                      0            0       0
##   Birman                                      0            0       1
##   Bombay                                      0            0       0
##   Boxer                                       0            0       0
##   British Shorthair                           0            0       0
##   Chihuahua                                   7            0       0
##   Egyptian Mau                                0            0       0
##   English Setter                              0            0       0
##   German shorthaired                          0            0       0
##   Great Pyreness                              0            0       0
##   Havanese                                    0            1       0
##   Japanese Chin                               0            0       0
##   Keeshond                                    0            0       0
##   Leonberger                                  0            0       0
##   Maine Coon                                  0            0       4
##   Miniature Pinscher                         74            0       0
##   Newfoundland                                0           79       0
##   Persian                                     0            0      73
##   Pomeranian                                  0            0       0
##   Pug                                         0            0       0
##   Ragdoll                                     0            0       2
##   Russian Blue                                0            0       0
##   Saint Bernard                               0            0       0
##   Samoyed                                     0            0       0
##   Scottish Terrier                            0            1       0
##   Shiba Inu                                   1            0       0
##   Siamese                                     0            0       0
##   Sphynx                                      0            0       0
##   Staffordshire Bull Terrier                  0            0       0
##   Wheaten Terrier                             0            0       0
##   Yorkshire Terrier                           1            0       0
##                             prediction
## value                        Pomeranian Pug Ragdoll Russian Blue
##   Abyssinian                          0   0       0            0
##   American bulldog                    0   0       0            0
##   American Pit Bull Terrier           0   0       0            0
##   Basset Hound                        0   0       0            0
##   Beagle                              0   0       0            0
##   Bengal                              0   0       0            0
##   Birman                              0   0       5            0
##   Bombay                              0   0       0            1
##   Boxer                               0   0       0            0
##   British Shorthair                   0   0       1            9
##   Chihuahua                           0   0       0            0
##   Egyptian Mau                        0   0       0            1
##   English Setter                      0   0       0            0
##   German shorthaired                  0   0       0            0
##   Great Pyreness                      0   0       0            0
##   Havanese                            0   0       0            0
##   Japanese Chin                       0   0       0            0
##   Keeshond                            0   0       0            0
##   Leonberger                          0   0       0            0
##   Maine Coon                          0   0       6            0
##   Miniature Pinscher                  0   0       0            0
##   Newfoundland                        0   0       0            0
##   Persian                             0   0       4            0
##   Pomeranian                         79   0       0            0
##   Pug                                 0  76       0            0
##   Ragdoll                             0   0      58            0
##   Russian Blue                        0   0       0           71
##   Saint Bernard                       0   0       0            0
##   Samoyed                             0   0       0            0
##   Scottish Terrier                    0   0       0            0
##   Shiba Inu                           0   0       0            0
##   Siamese                             0   0       1            0
##   Sphynx                              0   0       0            0
##   Staffordshire Bull Terrier          0   0       0            0
##   Wheaten Terrier                     1   0       0            0
##   Yorkshire Terrier                   0   0       0            0
##                             prediction
## value                        Saint Bernard Samoyed Scottish Terrier
##   Abyssinian                             0       0                0
##   American bulldog                       2       0                0
##   American Pit Bull Terrier              0       0                0
##   Basset Hound                           0       0                0
##   Beagle                                 1       0                0
##   Bengal                                 0       0                0
##   Birman                                 0       0                0
##   Bombay                                 0       0                0
##   Boxer                                  0       0                0
##   British Shorthair                      0       0                0
##   Chihuahua                              0       0                0
##   Egyptian Mau                           0       0                0
##   English Setter                         1       0                0
##   German shorthaired                     0       0                0
##   Great Pyreness                         0       0                0
##   Havanese                               0       0                1
##   Japanese Chin                          0       0                0
##   Keeshond                               0       0                0
##   Leonberger                             0       0                0
##   Maine Coon                             0       0                0
##   Miniature Pinscher                     0       0                0
##   Newfoundland                           0       0                0
##   Persian                                0       0                0
##   Pomeranian                             0       1                0
##   Pug                                    0       0                0
##   Ragdoll                                0       0                0
##   Russian Blue                           0       0                0
##   Saint Bernard                         78       0                0
##   Samoyed                                0      74                1
##   Scottish Terrier                       0       0               75
##   Shiba Inu                              0       0                0
##   Siamese                                0       0                0
##   Sphynx                                 0       0                0
##   Staffordshire Bull Terrier             0       1                0
##   Wheaten Terrier                        0       0                1
##   Yorkshire Terrier                      0       0                0
##                             prediction
## value                        Shiba Inu Siamese Sphynx
##   Abyssinian                         0       0      3
##   American bulldog                   0       0      0
##   American Pit Bull Terrier          0       0      0
##   Basset Hound                       0       0      0
##   Beagle                             0       0      0
##   Bengal                             0       1      0
##   Birman                             0       1      0
##   Bombay                             0       0      0
##   Boxer                              0       0      0
##   British Shorthair                  0       0      0
##   Chihuahua                          1       0      0
##   Egyptian Mau                       0       0      0
##   English Setter                     0       0      0
##   German shorthaired                 0       0      0
##   Great Pyreness                     0       0      0
##   Havanese                           0       0      0
##   Japanese Chin                      0       0      0
##   Keeshond                           0       0      0
##   Leonberger                         0       0      0
##   Maine Coon                         0       0      0
##   Miniature Pinscher                 0       0      0
##   Newfoundland                       0       0      0
##   Persian                            0       0      0
##   Pomeranian                         0       0      0
##   Pug                                0       0      0
##   Ragdoll                            0       1      0
##   Russian Blue                       0       0      0
##   Saint Bernard                      0       0      0
##   Samoyed                            0       0      0
##   Scottish Terrier                   0       1      0
##   Shiba Inu                         78       0      0
##   Siamese                            0      59      0
##   Sphynx                             0       0     79
##   Staffordshire Bull Terrier         0       0      0
##   Wheaten Terrier                    0       0      0
##   Yorkshire Terrier                  0       0      0
##                             prediction
## value                        Staffordshire Bull Terrier Wheaten Terrier
##   Abyssinian                                          0               0
##   American bulldog                                    0               0
##   American Pit Bull Terrier                           8               0
##   Basset Hound                                        0               0
##   Beagle                                              1               0
##   Bengal                                              0               0
##   Birman                                              0               0
##   Bombay                                              0               0
##   Boxer                                               1               0
##   British Shorthair                                   0               0
##   Chihuahua                                           0               0
##   Egyptian Mau                                        0               0
##   English Setter                                      0               0
##   German shorthaired                                  0               0
##   Great Pyreness                                      0               0
##   Havanese                                            0               2
##   Japanese Chin                                       0               0
##   Keeshond                                            0               0
##   Leonberger                                          0               0
##   Maine Coon                                          0               0
##   Miniature Pinscher                                  1               0
##   Newfoundland                                        0               0
##   Persian                                             0               0
##   Pomeranian                                          0               0
##   Pug                                                 0               0
##   Ragdoll                                             0               0
##   Russian Blue                                        0               0
##   Saint Bernard                                       0               0
##   Samoyed                                             0               0
##   Scottish Terrier                                    1               0
##   Shiba Inu                                           0               1
##   Siamese                                             0               0
##   Sphynx                                              0               0
##   Staffordshire Bull Terrier                         59               0
##   Wheaten Terrier                                     0              72
##   Yorkshire Terrier                                   0               0
##                             prediction
## value                        Yorkshire Terrier
##   Abyssinian                                 0
##   American bulldog                           0
##   American Pit Bull Terrier                  0
##   Basset Hound                               0
##   Beagle                                     0
##   Bengal                                     1
##   Birman                                     0
##   Bombay                                     0
##   Boxer                                      0
##   British Shorthair                          0
##   Chihuahua                                  0
##   Egyptian Mau                               0
##   English Setter                             0
##   German shorthaired                         0
##   Great Pyreness                             0
##   Havanese                                   2
##   Japanese Chin                              0
##   Keeshond                                   0
##   Leonberger                                 0
##   Maine Coon                                 0
##   Miniature Pinscher                         0
##   Newfoundland                               0
##   Persian                                    0
##   Pomeranian                                 0
##   Pug                                        0
##   Ragdoll                                    0
##   Russian Blue                               0
##   Saint Bernard                              0
##   Samoyed                                    0
##   Scottish Terrier                           2
##   Shiba Inu                                  0
##   Siamese                                    0
##   Sphynx                                     0
##   Staffordshire Bull Terrier                 0
##   Wheaten Terrier                            2
##   Yorkshire Terrier                         76

These images have the highest probabilities of being in a class:

set.seed(1)
par(mfrow = c(2, 3))

y_pred_mat <- predict(model, X)
id <- apply(y_pred_mat, 2, which.max)

for (i in id) try({
  par(mar = rep(0, 4L))
  plot(0,0,xlim=c(0,1),ylim=c(0,1),axes= FALSE,type = "n")
  Z <- image_to_array(image_load(pets$path_to_image[i], target_size = c(224,224)))
  rasterImage(Z /255,0,0,1,1)
  text(0.5, 0.1, label = class_names[y_pred[i] + 1L], col = "red", cex=2)
})

These are 30 images that were misclassified from the validation sample set. Due to varying degrees of light in these images, we believe light obfuscates the difference between two clearly different classes. Further avenue to improve the model should be able to account for the lighting effect.

#class_vector <- pets$class_name
class_names <- levels(factor(pets$class_name))

set.seed(1)
par(mfrow = c(2, 3))
id <- sample(which(y_pred != y), 30)

for (i in id) {
  par(mar = rep(0, 4L))
  plot(0,0,xlim=c(0,1),ylim=c(0,1),axes= FALSE,type = "n")
  Z <- image_to_array(image_load(pets$path_to_image[i], target_size = c(224,224)))
  rasterImage(Z /255,0,0,1,1)
  text(0.5, 0.1, label = class_names[y_pred[i] + 1L], col = "red", cex=2)
}

We try to visualize the embedding itself using principle components:

pca <- as_tibble(prcomp(X)$x[,1:2])
pca$y <- class_names[y + 1L]

We now plot it:

ggplot(pca, aes(PC1, PC2)) +
  geom_point(aes(color = y), alpha = 0.2, size = 4) +
  labs(x = "", y = "", color = "class") +
  theme_minimal()